﻿import traceback
import typing
import configparser
import requestNovelsLib
import datetime
import lxml
import lxml.etree
import requests
import multiprocessing
import multiprocessing.managers
import multiprocessing.synchronize

## %Y-%m-%d %H:%M:%S
timeFmt = "(%m-%d)"

__runMain = False

__maxNovelCount = 200
_pageinfocount = 20000
__var: multiprocessing.managers.ValueProxy[int] = None
__typeCount: multiprocessing.managers.ValueProxy[int] = None
__novelMapCount: multiprocessing.managers.ValueProxy[dict[str, int]] = None


def getNovelHome(response: requests.Response) -> typing.Union[dict[str, list[str]], None]:
	"""
	首页查找到类型与 url 的映射
	:param response: get请求之后而的响应体
	:return: 类型名称与之匹配的 url 映射
	"""
	xpath = requestNovelsLib.XpathHtml(requestNovelsLib.getAutoCodeText(response))
	## 获取 li 节点
	liNodes = xpath.xpath(r'//div[@class="nav"]/ul/li/a')
	if xpath.resultLen() == 0:
		return
	## 类型名过滤器
	getTypeName = requestNovelsLib.getNovelTypeName()
	
	## 返回
	result: dict[str, list[str]] = {}
	## 遍历需要的类型名称
	typeNameUrl = ""
	for liNode in liNodes:
		liNodeXpathInfo = requestNovelsLib.XpathInfo(liNode)
		aTests = liNodeXpathInfo.xpath(r'./text()')
		if liNodeXpathInfo.resultLen() == 0:
			continue
		typeName = aTests[0]
		typeName = requestNovelsLib.strRemoveSpaces(typeName)
		for item in getTypeName:
			if item == typeName:
				href = liNodeXpathInfo.xpath(r'./@href')[0]
				href = href.replace(response.url, "")[1:]
				typeNameUrl = requestNovelsLib.getResponseAtStr(response, href)
				break
		if len(typeNameUrl) != 0:
			if result.get(typeName) is None:
				result[typeName] = [typeNameUrl]
			else:
				result[typeName].append(typeNameUrl)
			typeNameUrl = ""
	return result


def getPageListNovel(response: requests.Response, getCount: int, novelTypeName, novelTypeUrl, novelRootUrl) -> typing.Union[list[requestNovelsLib.NovelInfo], None]:
	"""
	获取小说信息
	若为 none 或 数量 0， 则退出这次类型的扫描
	:param response: get 之后的响应体
	:param getCount: 已经库存的数量
	:param novelTypeName: 小说类型名称
	:param novelTypeUrl: 小说类型 url
	:param novelRootUrl: 小说网站
	:return: 小说列表
	"""
	global __maxNovelCount
	xpath = requestNovelsLib.XpathHtml(requestNovelsLib.getAutoCodeText(response))
	liNodes = xpath.xpath(r'//div[@class="l"]/ul/li')
	if xpath.resultLen() == 0:
		return None
	## 小说网址
	novelUrl: str = ""
	## 小说名称
	novelName: str = ""
	## 小说作者
	author: str = ""
	## 最后更新日期
	updateTime: str = ""
	## 最后更新选项
	lastItem: str = ""
	## 小说详情
	info: str = ""
	## 小说页面请求
	novelInfoPageRequestResult = None
	## 小说页面请求xpath
	novelInfoPageXpathResult = None
	## 返回值
	result: list[requestNovelsLib.NovelInfo] = []
	novelItemCount = 0
	removeTimeStrLen = len("最后更新：")
	for liNode in liNodes:
		novelUrl = novelName = novelName = author = updateTime = lastItem = info = ""
		novelInfoPageRequestResult = None
		novelInfoPageXpathResult = None
		
		try:
			## 小说 url 链接
			nodeXpathInfo = requestNovelsLib.XpathInfo(liNode)
			nodeTextArray = nodeXpathInfo.xpath(r'.//span[@class="s2"]/a/@href')
			if nodeXpathInfo.resultLen() == 0:
				print(f"发现 [{novelTypeName} ]  : xxx ==== >  异常")
				continue
			novelUrl = nodeTextArray[0]
			
			## 小说最后时间
			nodeTextArray = nodeXpathInfo.xpath(r'.//span[@class="s3"]/text()')
			if nodeXpathInfo.resultLen() == 0:
				print(f"发现 [{novelTypeName} ]  : {novelUrl} ==== > 小说最后时间 异常")
				continue
			updateTime = nodeTextArray[0]
			## 时间跳出
			if not requestNovelsLib.checkUpDate(updateTime, timeFmt):
				print(f"发现 [{novelTypeName} ]  : {novelUrl} ==== > 小说最后时间 超出时间")
				continue
			## 小说名称
			nodeTextArray = nodeXpathInfo.xpath(r'.//span[@class="s2"]/a/text()')
			if nodeXpathInfo.resultLen() == 0:
				print(f"发现 [{novelTypeName} ]  : {novelUrl} ==== > 小说名称 异常")
				continue
			novelName = nodeTextArray[0]
			## 小说最后更新
			nodeTextArray = nodeXpathInfo.xpath(r'.//span[@class="s3"]/a/text()')
			if nodeXpathInfo.resultLen() == 0:
				print(f"发现 [{novelTypeName} ] {novelName} : {novelUrl} ==== > 小说最后更新 异常")
				continue
			lastItem = nodeTextArray[0]
			## 小说作者
			nodeTextArray = nodeXpathInfo.xpath(r'.//span[@class="s5"]/text()')
			if nodeXpathInfo.resultLen() == 0:
				print(f"发现 [{novelTypeName} ] {novelName} : {novelUrl} ==== > 小说作者 异常")
				continue
			author = nodeTextArray[0]
			## 小说介绍
			# nodeTextArray = novelInfoPageXpathResult.xpath(r'//div[@id="intro"]/p/text()')
			# if novelInfoPageXpathResult.resultLen() == 0:
			# 	print(f"发现 [{novelTypeName} ] {novelName} : {novelUrl} ==== > 小说介绍 异常")
			# 	continue
			# info = nodeTextArray[0]
			# info = info.strip()
			
			print(f"发现 [{novelTypeName} ({getCount})] {novelName} : {novelUrl} ==== >")
			## 添加单个对象
			result.append(requestNovelsLib.NovelInfo(novelRootUrl=novelRootUrl, novelTypeName=novelTypeName, novelTypeUrl=novelTypeUrl, novelUrl=novelUrl, novelName=novelName, author=author, updateTime=updateTime, lastItem=lastItem, info=info, updateTimeFmt=timeFmt))
			getCount += 1
			if __maxNovelCount is None:
				__maxNovelCount = int(requestNovelsLib.readIni(requestNovelsLib.getDownLoadIniFilePath(), f"{getRootUrl()}", "count", __defMaxNovelCount))
			if 0 < __maxNovelCount < getCount:
				break
		except:
			requestNovelsLib.writeLogFile(traceback.format_exc(), response.url, response)
			traceback.print_exc()
			print(f"================>\n\t{novelUrl}\n<===========")
	## 返回
	if len(result) == 0:
		return None
	return result


typeCountMap = {}


def getPageNextUrl(response: requests.Response, exsiNovel: int, noveList, appenLen, novelTypeName: str, novelTypeUrl: str, novelRootUrl: str) -> typing.Union[str, None]:
	"""
	获取小说翻页的下一页
	:param response: get 之后的响应体
	:param exsiNovel: 已经库存的小说
	:param noveList: 查找到的小说
	:param appenLen: 新增数量
	:param novelTypeName: 小说类型名称
	:param novelTypeUrl: 小说类型 url
	:param novelRootUrl: 小说网站
	:return: 下一页，不存在则返回 none
	"""
	global __maxNovelCount
	if __maxNovelCount > 0:
		typeCount = 0
		try:
			typeCount = typeCountMap.get(novelTypeName)
			if typeCount is None:
				typeCount = 0
		except:
			traceback.print_exc()
		if appenLen > __maxNovelCount:
			return None
		elif typeCount > __maxNovelCount:
			return None
		typeCountMap[novelTypeName] = typeCount + appenLen
	if appenLen == 0:
		return None
	xpath = requestNovelsLib.XpathHtml(response.text)
	
	nextPage = xpath.xpath(r'//div[@id="pagelink"]/a[@class="next"]/@href')
	if xpath.resultLen() == 0:
		return None
	endPage = xpath.xpath(r'//div[@id="pagelink"]/a[@class="ngroup"]/@href')
	if xpath.resultLen() == 0:
		return None
	if nextPage[0] == endPage[0]:
		return None
	return nextPage[0]


def sort(novel: requestNovelsLib.NovelInfo):
	return novel.updateTime


__runTime = None


def setRunTime(time):
	global __runTime
	__runTime = time


def getRunTime():
	global __runTime
	return __runTime


def getRootUrl():
	return 'www.qianyege.com'


def modeRequestGetUrl(lock: multiprocessing.synchronize.Lock = None, workerThreadNumber=0, nextRequestTimeSece=0.05, titleSleepTime=10):
	global __maxNovelCount, __runMain, __var, __typeCount, __novelMapCount, _pageinfocount
	
	if not __runMain:
		if lock is None:
			raise RuntimeError("请在调用时候设置它的进程锁")
	else:
		if lock is None:
			lock = multiprocessing.Lock()
	
	__var = multiprocessing.Value('i', 0)
	__typeCount = multiprocessing.Value('i', 0)
	manager = multiprocessing.Manager()
	__novelMapCount = manager.dict()
	try:
		__maxNovelCount = int(requestNovelsLib.readIni(requestNovelsLib.getDownLoadIniFilePath(), f"{getRootUrl()}", "count", __maxNovelCount))
	except:
		requestNovelsLib.writeLogFile("无法获取匹配的数量", httpUrl=getRootUrl())
		traceback.print_exc()
	try:
		_pageinfocount = int(requestNovelsLib.readIni(requestNovelsLib.getDownLoadIniFilePath(), f"{getRootUrl()}", "pageinfocount", _pageinfocount))
	except:
		requestNovelsLib.writeLogFile("无法获取匹配的数量", httpUrl=getRootUrl())
		traceback.print_exc()
	rootUrl = f"https://{getRootUrl()}/"
	requestUrl = requestNovelsLib.ResponseNovel(rootUrl)
	newFolder = requestNovelsLib.getUrlTargetFolder(getRootUrl())
	try:
		## 获取类型类型的请求页面
		runEndTime = requestNovelsLib.ayscRequestGetUrl(requestUrl, getNovelHome, getPageListNovel, getPageNextUrl, rootUrl, newFolder, __runMain, proLock=lock, nextRequestTimeSece=nextRequestTimeSece, workerThreadNumber=workerThreadNumber, titleSleepTime=titleSleepTime, novelMaxCount=__maxNovelCount)
		setRunTime(runEndTime)
	except:
		traceback.print_exc()
	return


if __name__ == '__main__':
	__runMain = True
	requestNovelsLib.initLib()
	modeRequestGetUrl()
